Overview

This notebook gives a general overview of the features included in the dataset.

Show imports Hide imports

%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils

RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("c_schumann_lieder", corpus_release="v2.4")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Clara Schumann – Lieder version v2.4")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D

Data and software versions
--------------------------

Clara Schumann – Lieder version v2.4
Datapackage 'c_schumann_lieder' @ v2.4
dimcat version 3.4.0

Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'c_schumann_lieder': ["'c_schumann_lieder.measures' (MuseScoreFacetName.MuseScoreMeasures)",
                                               "'c_schumann_lieder.notes' (MuseScoreFacetName.MuseScoreNotes)",
                                               "'c_schumann_lieder.expanded' (MuseScoreFacetName.MuseScoreHarmonies)",
                                               "'c_schumann_lieder.chords' (MuseScoreFacetName.MuseScoreChords)",
                                               "'c_schumann_lieder.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}

filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata

		TimeSig	KeySig	last_mc	last_mn	length_qb	last_mc_unfolded	last_mn_unfolded	length_qb_unfolded	volta_mcs	all_notes_qb	...	wikidata	musicbrainz	viaf	pdf	staff_1_ambitus	staff_1_instrument	staff_2_ambitus	staff_2_instrument	staff_3_ambitus	staff_3_instrument
corpus	piece
c_schumann_lieder	op13no1 Ich stand in dunklen Traumen	{1: '3/4'}	{1: -3}	37	37	111.0	37	37	111.0	()	552.50	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/6191b47b-f0e0-475...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	62-77 (D4-F5)	Voice	55-79 (G3-G5)	Piano	34-65 (Bb1-F4)	Piano
	op13no2 Sie liebten sich beide	{1: '6/8'}	{1: -2}	34	34	102.0	34	34	102.0	()	445.00	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/1900782d-8c3d-4bb...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	62-75 (D4-Eb5)	Voice	55-80 (G3-Ab5)	Piano	31-65 (G1-F4)	Piano
	op13no3 Liebeszauber	{1: '4/4'}	{1: -3}	55	54	216.5	55	54	216.5	()	1103.17	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/1e4cb31b-6e23-4d9...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-79 (Eb4-G5)	Voice	53-79 (F3-G5)	Piano	29-63 (F1-Eb4)	Piano
	op13no4 Der Mond kommt still gegangen	{1: '6/8'}	{1: -5}	33	33	99.0	33	33	99.0	()	412.50	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/6916a104-8693-4d3...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	61-77 (Db4-F5)	Voice	58-78 (Bb3-Gb5)	Piano	32-65 (Ab1-F4)	Piano
	op13no5 Ich hab in deinem Auge	{1: '3/4'}	{1: -4}	34	33	100.5	34	33	100.5	()	489.00	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/74125f0c-d415-496...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-77 (Eb4-F5)	Voice	55-87 (G3-Eb6)	Piano	27-72 (Eb1-C5)	Piano
	op13no6 Die stille Lotosblume	{1: '3/4'}	{1: -4}	47	47	141.0	47	47	141.0	()	708.83	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/8ac06705-0692-49d...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-77 (Eb4-F5)	Voice	53-80 (F3-Ab5)	Piano	27-68 (Eb1-Ab4)	Piano
	op23no1 Was weinst du Blumlein	{1: '2/4'}	{1: 3}	69	68	136.5	69	68	136.5	()	498.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/89c905db-3e4d-4ae...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	61-81 (C#4-A5)	Voice	52-81 (E3-A5)	Piano	33-66 (A1-F#4)	Piano
	op23no2 An einem lichten Morgen	{1: '4/4'}	{1: -1}	39	38	156.0	72	72	288.0	(35], [36)	319.70	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/f2ea76f6-e65c-4ad...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-81 (F4-A5)	Voice	58-89 (Bb3-F6)	Piano	34-89 (Bb1-F6)	Piano
	op23no3 Geheimes Flustern hier und dort	{1: '3/8'}	{1: -5}	56	50	84.0	94	94	141.0	()	234.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/05e86d18-cf58-4f8...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-78 (F4-Gb5)	Voice	44-78 (Ab2-Gb5)	Piano	33-68 (A1-Ab4)	Piano
	op23no4 Auf einem grunen Hugel	{1: '2/4'}	{1: 0}	32	30	62.5	45	44	88.5	(17], [18)	248.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/79bbbe7d-4f93-445...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	68-79 (G#4-G5)	Voice	57-77 (A3-F5)	Piano	45-64 (A2-E4)	Piano
	op23no5 Das ist ein Tag der klingen mag	{1: '6/8'}	{1: 2}	46	45	135.5	46	45	135.5	()	512.00	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/edc5298a-7e55-4e5...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	64-81 (E4-A5)	Voice	54-81 (F#3-A5)	Piano	32-81 (G#1-A5)	Piano
	op23no6 O Lust o Lust	{1: '6/8'}	{1: -3}	40	40	120.0	76	76	228.0	()	416.75	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/17971c9f-d201-456...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-79 (F4-G5)	Voice	53-82 (F3-Bb5)	Piano	29-72 (F1-C5)	Piano

12 rows × 60 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}

mean_composition_years

corpus
c_schumann_lieder    1847.75
Name: mean_composition_year, dtype: float64

Composition dates

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)

Composition dates range from 1842 ('c_schumann_lieder', 'op13no3 Liebeszauber') to 1853 ('c_schumann_lieder', 'op23no1 Was weinst du Blumlein').

Mean composition years per corpus

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()

Show source Hide source

summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()

summary

		TimeSig	KeySig	last_mc	last_mn	length_qb	last_mc_unfolded	last_mn_unfolded	length_qb_unfolded	volta_mcs	all_notes_qb	...	wikidata	musicbrainz	viaf	pdf	staff_1_ambitus	staff_1_instrument	staff_2_ambitus	staff_2_instrument	staff_3_ambitus	staff_3_instrument
corpus	piece
c_schumann_lieder	op13no1 Ich stand in dunklen Traumen	{1: '3/4'}	{1: -3}	37	37	111.0	37	37	111.0	()	552.50	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/6191b47b-f0e0-475...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	62-77 (D4-F5)	Voice	55-79 (G3-G5)	Piano	34-65 (Bb1-F4)	Piano
	op13no2 Sie liebten sich beide	{1: '6/8'}	{1: -2}	34	34	102.0	34	34	102.0	()	445.00	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/1900782d-8c3d-4bb...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	62-75 (D4-Eb5)	Voice	55-80 (G3-Ab5)	Piano	31-65 (G1-F4)	Piano
	op13no3 Liebeszauber	{1: '4/4'}	{1: -3}	55	54	216.5	55	54	216.5	()	1103.17	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/1e4cb31b-6e23-4d9...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-79 (Eb4-G5)	Voice	53-79 (F3-G5)	Piano	29-63 (F1-Eb4)	Piano
	op13no4 Der Mond kommt still gegangen	{1: '6/8'}	{1: -5}	33	33	99.0	33	33	99.0	()	412.50	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/6916a104-8693-4d3...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	61-77 (Db4-F5)	Voice	58-78 (Bb3-Gb5)	Piano	32-65 (Ab1-F4)	Piano
	op13no5 Ich hab in deinem Auge	{1: '3/4'}	{1: -4}	34	33	100.5	34	33	100.5	()	489.00	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/74125f0c-d415-496...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-77 (Eb4-F5)	Voice	55-87 (G3-Eb6)	Piano	27-72 (Eb1-C5)	Piano
	op13no6 Die stille Lotosblume	{1: '3/4'}	{1: -4}	47	47	141.0	47	47	141.0	()	708.83	...	https://www.wikidata.org/wiki/Q93451955	https://musicbrainz.org/work/8ac06705-0692-49d...	https://viaf.org/viaf/175203352/	https://imslp.org/wiki/Special:ReverseLookup/3...	63-77 (Eb4-F5)	Voice	53-80 (F3-Ab5)	Piano	27-68 (Eb1-Ab4)	Piano
	op23no1 Was weinst du Blumlein	{1: '2/4'}	{1: 3}	69	68	136.5	69	68	136.5	()	498.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/89c905db-3e4d-4ae...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	61-81 (C#4-A5)	Voice	52-81 (E3-A5)	Piano	33-66 (A1-F#4)	Piano
	op23no2 An einem lichten Morgen	{1: '4/4'}	{1: -1}	39	38	156.0	72	72	288.0	(35], [36)	319.70	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/f2ea76f6-e65c-4ad...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-81 (F4-A5)	Voice	58-89 (Bb3-F6)	Piano	34-89 (Bb1-F6)	Piano
	op23no3 Geheimes Flustern hier und dort	{1: '3/8'}	{1: -5}	56	50	84.0	94	94	141.0	()	234.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/05e86d18-cf58-4f8...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-78 (F4-Gb5)	Voice	44-78 (Ab2-Gb5)	Piano	33-68 (A1-Ab4)	Piano
	op23no4 Auf einem grunen Hugel	{1: '2/4'}	{1: 0}	32	30	62.5	45	44	88.5	(17], [18)	248.25	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/79bbbe7d-4f93-445...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	68-79 (G#4-G5)	Voice	57-77 (A3-F5)	Piano	45-64 (A2-E4)	Piano
	op23no5 Das ist ein Tag der klingen mag	{1: '6/8'}	{1: 2}	46	45	135.5	46	45	135.5	()	512.00	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/edc5298a-7e55-4e5...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	64-81 (E4-A5)	Voice	54-81 (F#3-A5)	Piano	32-81 (G#1-A5)	Piano
	op23no6 O Lust o Lust	{1: '6/8'}	{1: -3}	40	40	120.0	76	76	228.0	()	416.75	...	https://www.wikidata.org/wiki/Q97192858	https://musicbrainz.org/work/17971c9f-d201-456...	https://viaf.org/viaf/317275603/	https://imslp.org/wiki/Special:ReverseLookup/4...	65-79 (F4-G5)	Voice	53-82 (F3-Bb5)	Piano	29-72 (F1-C5)	Piano

12 rows × 60 columns

Composition years histogram

Show source Hide source

hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions

Overview

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)

	pieces	measures	length	notes	labels
6 Lieder aus Jucunde, Op. 23	6	271	694	3745	613
6 Lieder mit Begleitung des Pianoforte, op.13	6	238	770	6503	713
sum	12	509	1464	10248	1326

def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())

|                   |   pieces |   measures |   length |   notes |   labels |
|:------------------|---------:|-----------:|---------:|--------:|---------:|
| c_schumann_lieder |       12 |        509 |     1464 |   10248 |     1326 |
| sum               |       12 |        509 |     1464 |   10248 |     1326 |

Measures

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()

520 measures over 12 files.

			mc	mn	quarterbeats	duration_qb	keysig	timesig	act_dur	mc_offset	numbering_offset	dont_count	barline	breaks	repeats	next	volta
corpus	piece	i
c_schumann_lieder	op13no1 Ich stand in dunklen Traumen	0	1	1	0	3.0	-3	3/4	3/4	0	<NA>	<NA>	<NA>	<NA>	firstMeasure	(2,)	<NA>
		1	2	2	3	3.0	-3	3/4	3/4	0	<NA>	<NA>	<NA>	<NA>	<NA>	(3,)	<NA>
		2	3	3	6	3.0	-3	3/4	3/4	0	<NA>	<NA>	<NA>	<NA>	<NA>	(4,)	<NA>
		3	4	4	9	3.0	-3	3/4	3/4	0	<NA>	<NA>	<NA>	line	<NA>	(5,)	<NA>
		4	5	5	12	3.0	-3	3/4	3/4	0	<NA>	<NA>	<NA>	<NA>	<NA>	(6,)	<NA>

all_measures.get_default_analysis().plot_grouped()

Harmony labels

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")

			mc	mn	quarterbeats	duration_qb	mc_onset	mn_onset	timesig	staff	voice	volta	...	numeral_or_applied_to_numeral	intervals_over_bass	intervals_over_root	scale_degrees	scale_degrees_and_mode	scale_degrees_major	scale_degrees_minor	globalkey	localkey	chord
corpus	piece	i
c_schumann_lieder	op13no1 Ich stand in dunklen Traumen	0	1	1	0	2.0	0	0	3/4	3	1	<NA>	...	I	(M3, P5)	(M3, P5)	(1, 3, 5)	(1, 3, 5), major	(1, 3, 5)	(1, #3, 5)	Eb	I	I
		1	1	1	2	0.5	1/2	1/2	3/4	3	1	<NA>	...	ii	(M2, P4, m6)	(m3, d5, m7)	(1, 2, 4, b6)	(1, 2, 4, b6), major	(1, 2, 4, b6)	(1, 2, 4, 6)	Eb	I	ii%2
		2	1	1	5/2	0.5	5/8	5/8	3/4	3	1	<NA>	...	V	(m3, a4, M6)	(m3, d5, d7)	(1, b3, #4, 6)	(1, b3, #4, 6), major	(1, b3, #4, 6)	(1, 3, #4, #6)	Eb	I	viio43/V
		3	2	2	3	1.0	0	0	3/4	3	1	<NA>	...	V	(m3, m7)	(M3, P5, M2)	(7, 2, 6)	(7, 2, 6), major	(7, 2, 6)	(#7, 2, #6)	Eb	I	V6(2)
		4	2	2	4	2.0	1/4	1/4	3/4	3	1	<NA>	...	V	(m3, m6)	(M3, P5)	(7, 2, 5)	(7, 2, 5), major	(7, 2, 5)	(#7, 2, 5)	Eb	I	V6

5 rows × 52 columns

Concatenated annotation tables contains 1308 rows.
Dataset contains 1308 tokens and 247 types over 12 documents.